package my.news.crawler.msn_money;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.StringWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;

import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import com.aliasi.spell.EditDistance;
import com.porva.html.keycontent.Kce;
import com.porva.html.keycontent.KceSettings;
import com.porva.html.keycontent.LinkFoundListener;
import com.porva.html.keycontent.NewsXMLCleaner;
import com.porva.html.keycontent.TitleFoundListener;

import my.crawler.util.IOUtil;
import my.crawler.util.XMLUtil;
import my.news.Content;

public class MSNCrawler {
    static Logger LOG = Logger.getLogger(MSNCrawler.class.getName());

    public static class MSNNewsListPage {
        final String base = "http://news.moneycentral.msn.com/ticker/";

        URL current;
        MSNNewsListPage nextListPage;
        Document doc;
        int offset;
        String companyName;
        List<MSNNewsStory> stories = new ArrayList<MSNNewsStory>();

        public void load() throws Exception {
            current = new URL(String.format(base
                    + "rcnews.aspx?symbol=%s&view=all&ipage=%d", companyName,
                    offset));
            doc = XMLUtil.buildDOM(current);
//            List<Node> nodes;
//            nodes = XMLUtil.getNodes(doc, String.format(
//                    "//html:A/@href[contains(.,'ipage=%d')]", offset + 1));
//            // set next
//            if (nodes.size() > 0) {
//                nextListPage = new MSNNewsListPage(offset + 1, companyName);
//            }
            Node closest = getLinkWithClosestURL(current.toExternalForm());
            if(closest != null) {
                nextListPage = new MSNNewsListPage(offset + 1, companyName);
            }

            // get all links
            for (Node node : getLinks()) {
                String url = node.getAttributes().getNamedItem("href").getTextContent();
                if (!url.startsWith("http")) {
                    url = base + url;
                }
                stories.add(new MSNNewsStory(url));
            }
        }

        public MSNNewsListPage(int offset, String companyName) throws Exception {
            this.offset = offset;
            this.companyName = companyName;
        }

        public boolean hasNext() {
            return nextListPage != null;
        }

        public List<MSNNewsStory> getNewStroies() {
            return stories;
        }

        public MSNNewsListPage next() {
            return nextListPage;
        }

        public List<Node> getLinks() {
            Map<String,List<Node>> suspected = new HashMap<String, List<Node>>();
            for(Node n : XMLUtil.wrap(doc.getElementsByTagName("A"))) {
                String xpath = XMLUtil.getXPath(n);
                List<Node> list = suspected.get(xpath);
                if( list == null) {
                    list = new ArrayList<Node>();
                    suspected.put(xpath, list);
                }
                list.add(n);
            }

            List<Node> result = null;
            for(List<Node> list : suspected.values()) {
                if(list.size() > 30 || list.size()<10) {
                    continue;
                }
                double avgLength = 0;
                for(Node n : list) {
                    avgLength+=n.getTextContent().length();
                }
                avgLength/=list.size();
                if(avgLength < 20) {
                    continue;
                }
                if(result != null) {
                    throw new RuntimeException("Can't evaluate new links");
                }
                result = list;
            }
            return result;
        }

        public Node getLinkWithClosestURL(String currentURL) {
            EditDistance dist = new EditDistance(false);
            Node closest = null;
            double proximity = -Double.MAX_VALUE;
            for(Node n : XMLUtil.wrap(doc.getElementsByTagName("A"))) {
                String href = n.getAttributes().getNamedItem("href").getTextContent();
                double prox = dist.proximity(currentURL, href);
                if(prox>proximity && proximity != prox) {
                    proximity = prox;
                    closest = n;
                }
            }
            return closest;
        }

    }

    public static class MSNNewsStory {
        URL current;
        static Kce extractor;

        public Content content;
        static {
            // allocate a new extractor with default settings
            KceSettings settings = new KceSettings();
            // load settings from "conf/keycontent.properties" file
            settings.loadSettings("conf/keycontent.properties");
            // construct a new extractor
            extractor = new Kce(settings);
            // register additional listeners
            LinkFoundListener linkFoundListener = new LinkFoundListener();
            TitleFoundListener titleNodeFoundListener = new TitleFoundListener();
            extractor.registerNodeFoundListener(linkFoundListener);
            extractor.registerNodeFoundListener(titleNodeFoundListener);
        }

        public MSNNewsStory(String textContent) throws Exception {
            current = new URL(textContent);
        }

        public void load() throws Exception {
            String result = clean(IOUtil.getString(XMLUtil.openStream(current)));
            content = new Content(result);
        }

        public Content getContent() throws Exception {
            return content;
        }

        public String clean(String str) {
            Document document = extractor.extractKeyContent(
                    new ByteArrayInputStream(str.getBytes()), "ISO-8859-1",
                    null);
            StringWriter stringWriter = new StringWriter();
            // present cleaned document as String
            Kce.prettyPrint(document, "utf-8", stringWriter);
            return stringWriter.getBuffer().toString();

        }
    }
}
